{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import random\n",
    "import torch\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "from src.PPO.PPO import PPO\n",
    "\n",
    "# env = gym.make('CartPole-v1')\n",
    "env = gym.make('LunarLander-v2')\n",
    "env.seed(1234)\n",
    "\n",
    "state_size = env.observation_space.shape[0]\n",
    "action_size =env.action_space.n\n",
    "\n",
    "# PPO Settings\n",
    "update_every = 2000\n",
    "num_learn = 20\n",
    "win_condition = 200\n",
    "\n",
    "# Agent settings\n",
    "hidden_size=512\n",
    "epsilon=0.2\n",
    "entropy_beta=0.01\n",
    "gamma=0.99\n",
    "lr=0.002\n",
    "\n",
    "agent = PPO(state_size, action_size, hidden_size=hidden_size, epsilon=epsilon, entropy_beta=entropy_beta, gamma=gamma, lr=lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(n_episodes=2000, max_t=700):\n",
    "  steps = 0\n",
    "  scores_deque = deque(maxlen=100)\n",
    "  scores = []\n",
    "  average_scores = []\n",
    "  max_score = -np.Inf\n",
    "\n",
    "#   agent = PPO(state_size, action_size, hidden_size=hidden_size, epsilon=epsilon, entropy_beta=entropy_beta, gamma=gamma, lr=lr)\n",
    "\n",
    "  for episode in range(1, n_episodes+1):\n",
    "    state = env.reset()\n",
    "    score = 0\n",
    "    \n",
    "    for t in range(max_t):\n",
    "      steps += 1\n",
    "\n",
    "      action, log_prob = agent.act(torch.FloatTensor(state))\n",
    "      next_state, reward, done, _ = env.step(action.item())\n",
    "\n",
    "      agent.mem.add(torch.FloatTensor(state), action, reward, log_prob, done)\n",
    "\n",
    "      # Update \n",
    "      state = next_state\n",
    "      score += reward\n",
    "\n",
    "      if steps >= update_every:\n",
    "        agent.learn(num_learn)\n",
    "        agent.mem.clear()\n",
    "        steps = 0\n",
    "\n",
    "      if done:\n",
    "        break\n",
    "    \n",
    "    # Book Keeping\n",
    "    scores_deque.append(score)\n",
    "    scores.append(score)\n",
    "    average_scores.append(np.mean(scores_deque))\n",
    "      \n",
    "    if episode % 10 == 0:\n",
    "      print(\"\\rEpisode {}\tAverage Score: {:.2f}\tScore: {:.2f}\".format(episode, np.mean(scores_deque), score), end=\"\")\n",
    "    if episode % 100 == 0:\n",
    "      print(\"\\rEpisode {}\tAverage Score: {:.2f}\".format(episode, np.mean(scores_deque)))   \n",
    "    \n",
    "    if np.mean(scores_deque) > win_condition:\n",
    "      print(\"\\rEnvironment Solved in {} episodes!\".format(episode))\n",
    "      break\n",
    "\n",
    "\n",
    "  return scores, average_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 100\tAverage Score: -107.13\tScore: -45.24\n",
      "Episode 200\tAverage Score: -66.29\tScore: 20.1230\n",
      "Episode 300\tAverage Score: -30.95\tScore: -149.35\n",
      "Episode 400\tAverage Score: 187.67\tScore: 277.43\n",
      "Environment Solved in 409 episodes!\n"
     ]
    }
   ],
   "source": [
    "scores, average_scores = train(n_episodes=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x124f927b8>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD8CAYAAACCRVh7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzsnXecXFXd/99n+vbNpvfeAyHJkoReEnp9FKUpxZJHBBV/ooKoPAiiIqAoRSIgiGhAQAREQgKhk056IZves5vtfcr5/XHnzt6ZubO7s7uTnd39vl+vvDJz586dM7Mz53O+9SitNYIgCIJgh6OzByAIgiCkLyISgiAIQkJEJARBEISEiEgIgiAICRGREARBEBIiIiEIgiAkRERCEARBSIiIhCAIgpAQEQlBEAQhIa7OHkB76dOnjx4xYkRnD0MQBKFLsWrVqhKtdd+WzuvyIjFixAhWrlzZ2cMQBEHoUiildrfmPHE3CYIgCAkRkRAEQRASIiIhCIIgJEREQhAEQUiIiIQgCIKQEBEJQRAEISEiEoIgCEJCRCQEQejyBEOaF1fspboh0OZrlNY0crS6IXJ/R3E1dY3Bjhhel0ZEQhCELs8/lu/hRy+v46+f7mrzNabfs4gZ9y7mV29upqLWz9kPvs//e3FNh42xLeworuaht7cSCulOG4OIhCAIXZ5/rtwLQFV9gMOV9c2eu7e0lqIj1ZH7wZCOPB/giQ92cN+bmwH4aFtJCkYLFbV+rnt6OVsOVXKksp4Rt/+HT4riX+vaJ5fxh3eL2GYZ77FGREIQhC6N1joyiT7+3nZm3fcOwQQr77rGIFc+8SlzH3qfmrBr6vW1B/jhS+uizjtQUQdAljdx5yKtNY+/t529pbVJj/mJD7bzwefFvLBiL0t3lgLw3NLoLhlV9X4OVhiCt2ZvWdKv0VGISAhCO3n2k118uK24s4fRY6lqCFAbEztYtuNo5Pb24moeXVLEqt1lTPz5WxwIT7wvrNjL5oOVfGyzgi+taQQg25dYJPaU1vKbt7Zwy99X2z6utabebx/TWLTpMAChkKY+PPYMjzPqnI0HKiO3f/zyev6z7mDCsaQSEQlBaCd3vbaRrz61vLOH0S15edU+vvXcqmbPOVwR7156OzwJNwZCfOGxT/jtwq3c+Jfov9GB8jouePhD/rlqX9zzTZFwOVTC1zXPScRDiz5nws/e4lBFPW9vPMSVT3zKbxduAYhYCPvK6iirNa6T4Y4WCdNCmTAgB4C7X9/YKbGJLt8FVhCE7ssP/rkWgLKaRvIz3Xx+uJr8TDf9c30A1DYGWLDCiCc4FJhzqDmBbzhQQUWdH4DK+ujMp0AzE645iZfX+hOeUx6+bl6mBzAsh62HqxiUn8Hx//d25LzZv3oncnvZzlK+N2dcJAtrb1ktQwsyAeJcZPvK6lAKnv/GLF5fe4D/e30Tq/aUceKIgoRjSgViSQiC0Cl8tK2Eu1/f2Kpzp92ziAUr9nLe7z9g7kPvR44/vHgbT320E4CBeRmR45X1xgR+sNyY7K+eOSzumrWNLafLltY2onW8mJTWNFJabQhRfoYbgCc/3Mn5v/+QN9Y27xbaUWLET9xOxb6yOg6G4x9WQSqraWTFrlIG5Prone3l9HHGtg/7y+paHHNHIyIhCEIchfcu5t43NkUdK6tpbDFzKBm+8tQy/vLxrmbPGZzfNPG/stpwC1VZLIIay0TfO9sTuV0ZXuWbE/A1NiKx+2h0wPn7c8fFndMYCPHa2gPU+4PM/2A7C5bv4T/rDjL9nkUs3HgIgPxMQyTW7CsHDOugOd7eaLjCpgzOo7YxyIb9RuzBtHgAbnxmBZ9sP0qfbC8AeWEhKq9t3sWVCkQkBEGIo6S6gSfDK3STafcsYtZ97yR4RttpDIRYsuWIrb99YJ4vctsaA6gIr7pdjqYpzKGM+EHvLA+V4VTY5TtLyXA7mTI4l3Mm9Y+69q6jNZHb/XK8nDq2j+34vrdgDY+8W8R9b27h9lfW81GRkaTw3lbjf48zPIbw8O2iGH2yvbz0rZPIcDsj4nLaGOP19pcbQna4sj7ihtpebFgbprVjikRFXYAXVuxhwfI9BIIh2/F2NCISgiB0Ku9sPsyNz6zgN29tafa87cVNk/rGAxVAk3DceMqIyKp7SK8Mio5UM+u+d3h702EG5vlQSvGrLxwXdb3DlUZ19W3njuMf82ZHJmI75n+wI3L7aNjN1BiepM3YRijsllIWlTBvnzGuL4UjCpg+PD+StXTG+KadQzPcTnaU1DDlroWs3lPG1CH5ADzwpakAuJwOsr0uyusa+fHL67n9lfV8asngSiUiEoLQDuz81UJymJPtE5aJ2KQ+EGR03yy8LmOqys9043E6eGO94fcvq23khKH53HXJZO6/4njuvXxKXGC3tLb5TKVvnj6K0X2z40TijHFNk3hjMMRJo3oDcLiqIeo8fzDE6j1lbD1UFXftyYNy+dNXZvCLyyYDMGtk78hj/XKarKTLpw2K3H551T4agyFmjypg2rBekeN5Ge6IBQXR7qlUIiIhCO0gUdFWT8FaC/DQos8Zcft/knaDJKolAKP4bcKAXEb2yQJgZJ8srigcwksr9zHvryv5cFsJvbOMWERBloevzB4eN9mb0uBy2k937rDLKjcjOtmzd7aHuy6ZFLk/vLeRhVQSIxKBoOYLj33CjhLD0rFmTbkcDs6fMiBSlDdzZJOA9cry8K9vn8xj106n3t/0mTUGQviDIdwx483LcHPU4nKLrQ1JFSkVCaXUUKXUEqXUJqXURqXU98LHC5RSi5RS28L/9wofV0qpPyilipRS65RS01M5PkFoL82lUfYE/vLxLib87C1Kaxr50/vbgeQ/k+aa6NX7Q3jdhqsFoCDTw4kjetEYDEVqIXpleaKekxsWiYF5Pn71heN47uuzAHtLQilwhI97XU6+furIyGMepwOPq2mKNFNVi2MtiVC0KAaDVpGIfs0ThubjcTpwORRZHifThvXiwuMG0iuz6T00BkM0BkJNsY4w+ZluSiwNCI9V88FUWxIB4Ada60nAbOBmpdQk4HbgHa31WOCd8H2AC4Cx4X/zgMdTPD5BaBf+YxQ8TFdMv/hbGw5FjiXrgasPNH2Gse67en+QDLeTnHDlc68sT1SwGppSUE1MiyDL6+LqmcOYMjgPIG5lDk1WhMmdF06M3HY5VdRE3T/Xh8fliLjHTALB6DFHWRLOaJHwuZ2cMDSf/EwPyhK8uO28cfz9m7MY2y8bf9CwJKwCBYYlYRWJmlak8HYEKRUJrfVBrfXq8O0qYDMwGLgMeDZ82rPA5eHblwF/1QZLgXyl1MBUjlEQ2oN1gvj8cFXKGsIdS1oTZ3lg4VZ+/NI6JoargRdtOhTJ7tG03ZLwB+NFwud2kuMzhKAgyxM32ZfFFLzleI1zY/suOR0qKqgM8ZO4w6EwF//uGEvC63JEZVuZBGItiRh3UyzfmTOGW+eOjTqW6XFx8ug+hggFQviDOu599vGFyKzeiyLEGY61zNr62+QVuQ0cs4prpdQIYBqwDOivtTYrTg4BZm7aYGCv5Wn7wseiqlOUUvMwLA2GDYvPfxaEY4XV1XDu7z4AYNevL+qs4XQIrYmzPLKkCICbzhwNRKenJhunscYkrCtorTV1YUvCnMx7ZXpwx0zsI/tkRt03z82K6YUEhuVgtQScNi4ol9MRcfdYLQmPy0HvLE9cfUWssFlFI1aEAE4b25fTxsYdNsbndNAY1DQGLDGJPUth63/58dbnuMdzlBrtJUs1UF1cANWHIWeA/cU6iGMiEkqpbOBl4FatdaXVzNJaa6VUUt8qrfV8YD5AYWFhz3YKC51KdwxcB5NYnZpB6qDWEQsi2Y/EKhJWy8wf1IQ0+NyOyERckOWOCkCfPq4v3zpjdNT1TNfUcUPy4l7L5VRYXfn2LihFI/GWhMflwOeOF57YQH2gmZhES3hcDvyBEI3BED5nCBbeCZ8+AkB5fiFPl5xGAVUsD00gd8oXuTfFAgHHQCSUUm4MgXhea/1K+PBhpdRArfXBsDvpSPj4fmCo5elDwscEIS2J9Ud3B0IthFms7ihz8g6GmjwfyaYF11lEwrrKN4/73M7IRJzldeG2TLwnDM2Py1qaMbyAv9x4IqeOiS+Oi7Uc7CwJ85jLqaLdTc4EIhGjitY4lZ27qTk8Tgd1/iB9/Qf52q5fQdVKKPw6nHM3b68o4SFLFfxlgWOTnJpSkVCGyfAUsFlr/ZDlodeA64Ffh///t+X4LUqpBcAsoMLilhKEtKM7Bq5jfezxj1tFwjjXWi2dvLup6fWsr91gEQlTjFwOB27LxO1OsFI/a3w/2+OxK3u755ujdzsdUZaGYUnET8yxKbwNlkC808bdlJADa/hO6X30b9jJIL0fatxw6R9h+nXh1y+NOv1YpcCm2pI4BfgqsF4pZe4D+BMMcXhRKfV1YDfw5fBjbwIXAkVALXBjiscnCO2iO6bAtmRJRLuErO6m8POT/EisloQ/oOOOZ7idHD8kj8WbDzOkV0aUMLtdya2mVUzkOlHtBMSnwHpcDnyueEuipiGxSCQSsegLlMBr34Wt/2GKI5u1jsksDk2hccZNfHv6aZHTvDHvtTUNCjuClIqE1voj7FuZAMyxOV8DN6dyTILQkdhZElrruMmoK9GSJWEN1puCEQrpiJspWXdTVODacm3TwvC5ndx81hjOntCPKYPz2LC/InJOsj7/2LNtnx8efmwKrMflwGvjboqdrK3vpzkRQmvY8DIsugtqS+DMn3DX3pNYfTjEjuoavpcR3WsqNiW2u1gSgtCtsYtJBEI6LgOnK9FS4Nr6nhstlkRrnx+LXeA6FNL86CVjL4kMjwOnQ9nWO9gFnpPBLvvI6m7yRqXAOuPcTdleFzWNiS2JhCJWWwr/uBr2LoUBx8GX/wpDZhB6YQ01jUYadawoWMeS5XFS29A9iukEoVtjt+puDLQ+TqG15r/rDx6zjp6twS6mYLUOrGMNRALXHeRuCl/7SFUDa/cZFkOsi8c6sScrErFDczYTWLZ1N8VYErk+F7UNzVkSMSKhNSz/Mzx2EhxYbcQc5r0PQ2ZEXsN0X8VWXFvHkpfhptbfDYrpBKG7Y2dJJBPM/u+GQ9z0/Grb5nadhZ1I+G2sB+tta+A62S02rYFr87Oziq8vpt7BWiVtZwkkg53FZwqiy6miA9fO+JhEboa7BUsi/PzKg7DuRXj12/DmbZDdD770rBGUdjRd0+10RNqFx47N42w6Ly/Tc8zacoi7SRDagV3gOrZtQ3OYbRYOlB/7HccSYS8S8dYDNK2ag1pHUmBDyabA2lRcW0Up9nLRlkT7YhJ2KbBWd1O8JRG9rs61aS/eEGh6P8NqN8F//wqrnoVAnTGCU26FOXeBjRUT/XrRguR1Wy0JF7tKRCQEIe2xsxqScTeZAe50ypFqUSQsq3wzeGr9GNpXTBeKej23UzF5UG7U+dbVfbJ1CCZKGeIT27sp9nVi23LEupvs9qAwLaMBHOWrn98GoQaYcgWc8j0oGAnenGZfs+l2rCUR7W6q8wcJhXSkQWGqEJEQhHZg725q/Sxp/rzTaVuKltxN1ttmZo/VemhPWw7TCjOF9tFrpsdNzO52xCRMfC4ndf6gfeBaN72OJ9bdFGNJjFKHyKKe0x3rONOxlrOdn7HZP4pGN4xT+1Baw3dWQ+/oqvBEeCzjiQ1cW+/3zfHSJ9tDQyBEhk37kY5EREIQ2oFd4DqZmIS55WY6bV5kl52UyN3UZElY3UNtr7gORNxNYUvCpg7C1cxquyXMzGSv26hstnc36fC1o3s3ORwqSrC+5XyN27cv4I5wz7+gVmzRw+jPUfzKwV7djxWT7+KLrRQIiHE3NRO4/uZpo7j38uid9lKFiIQgtAM7qyE5d5PxfxppRIvBeGstgxlPiApcJ/lerOebomt+rrETJUQLQ7N1CM1gBKD9tpZIkyXhiHPl+FyKWWozt7j+xWnODezOnspb5UNYHprAe6ETCBK9qr+tYFxS42ouvdfTgam/ySAiIQjtwM61kkzguq3B3lRiN5YWLYl2uJvA8Pc3BEI0xloSLewBkawlYQ7TdBs1H7iOeWzXR5z69nc5z7udo6oXnPw9nm+4gvkf7Y27hkmyIma1FmKtKGudRHuzupJBUmAFoR3YuZb8SVgS6dj7qeWYRHwTvkCUJZG8SGSG/epm4LrREriOxbq6b+uK2hvOHLIVGYslAZBJPfe4noZnLsIZauSH/nlc7XsUzvkFTpcn/vkWkq0Ij025tRuz3WOpRCwJQWgHdimwyQSuzXTJ9LEjWs5ushO2UFRMIvnXzHA7KcMfubYptC2JQNJtOcKnm5aEXXaUGZPILtsM617mY+8/yKMGZt/M5+O/wz+fWM1gnWGMr4XXT7pVeFQbkJjsJpe4mwQhrSmtacTjatpvGeL3EgBoDLY+f92MX6STuymZOonIc9rRlgOIZOjE1knEZvjE0mZLIhyAtpvEC6jgy87FjPrXK+D08GFoGs8EzuOV87+P91AV0PT3asmd5GyPu6mZwPWxdDeJSAhCK5l+zyIKsjysuHMugVAIr8uZIHCdjCUR7n2URt1kbeMslvdkl9Fl1YW2CF6TSMTWSaRIJMITblRgWmtY9ic+cd2BU2lqxlxK1hf+yHfv/jhyimmBhCxV2c2Orz3uphiBtMZPmqvv6GgkJiEISVBa08i8v65k/E/fAtqfAmuKROyeBJ2JnSVgfZ8tudPaks6b6TbWq6aV0lxMwkryK2rjfDOVNfLsst3w7CXw1u18oKdyWcMvKL9oPmTkRz3bjAuYOtrSZJ1s4Lq1NSCpLqCzIpaEICTJO1uMjRRDId3uFFjz3IYknpNq7OMs9hXXdrQlFm/2Z/KHoi2JlgK0bV1RR0RCAXuXw4vXQ2MNXPw7vvlKPwJa2VoB5mrejMG0JFJt2b40cvsYxh2aIz1GIQhdkIo6f4v++5YwA9fpZEnYNeuzuptihTE2jbRN7qawG8fcdKi1gWu3q20rap/LgZdGLjz0ODx9ntFk78Y3ofBrhLSKeu0cn4uBeUbFXK9MN1OH5vPAl6cCLVsKyVo6sftXpANiSQhCGympbrANXLfN3XTsLIl/r9nPoPwMThxRYPu41ZIIao0DFeNuih6rz+WI6oTami6wsS4pr8uJUvHFdC3tPNeW3k0DOMr1++bzQ+8a+pWUw4wb4Jx7wGf0iIrUSYRf+7OfndP0ek4H/775lMj9Ds9usrxfu/20O4P0kCpBSFPqGoNM/vlbLNp0OO6xc373Ae9uPRJ3PBnXUWfEJL63YA1f+tOnCR+326+6ueym2N3aWhODjz3HbMvd2Io6CStJFdMd2czdwT+w0Ptjxld+wtbQEJ4a8SBc8nBEIKApCG9O8C6nI6HF0HLMoe2Ba7vmgZ2BiIQgNMOOkmpqGoM8+PZW28c37K+MO9ZSYPdodQNnP/ge24ur0z4mYd72R7mb4i0JK61xN8XGNdwOB26Hagpcm+6mFiyFVmU3+evhnV/An07lTL2MNaExPDv1b3zV/xO25cyMO33CgJxWX7ujdyA0kwZG9snq0Ou2B3E3CUIC9pfXsfGAIQLJmP4tuZsWbjzMjuIanvxwR0QcGtIpJmHTYqMxKnCtcTlUREBiLYnW1EnExnJcToXb5YhKgXU5VItZPC36/Et3wIvXwaH1MPUaLt1wDtvrvXwvaySwDbutyJ//xiy2HKqybdkRy9HqRgAG5vk4WFEf93iyW52P75/DaWP78NOLJiX3xBQiIiEINqzZW87ljzblx2ckIRLNZTc9+PZW/vhuEWCsVE1xqE8TS6Ki1h/lToqIRMDqbgrhtIpEjCXRmhTY2Awqt9OBy+GwFNOFWhW4TWhpBAOw/Al495fgdMPVL8D486nctBhoaPaavbO9nDLG2+JrAwzKNwLaX5w+hEeWFMU9nmwMP8vr4rmvz0ruSSlGRELolhQdqSakNeP6J97gpTn2l0XvFJdMz/7mLAlTIIAoH/yxiklYJ/B6fzDKQtpeXM2cB99n2rCm2gDTLVRUXE2O10VVQwB/0LAkzKk2LiZh8/YfWLgVl1Nx69xx4XNiLAmHwu1Ulk2HdKvcPbaWxvYl8NYdULwZxp4LFz0I+cMirwPWNijtcxedN3kAS++Yw57SWluR6A6kXUxCKXW+UmqrUqpIKXV7Z49H6JrMfeh9zv3dB21+fqzPPNaSuOyEQQmfu3RnKd/9x2ctup08LgcN/iaROBZ7SljdRkdrGqMe23LQaDnx2Z7yyDHzY9iwv4Ipg/NQCv6z/mCUSyk2JmHnbvrP+oO8+tn+yP1YS8LldOB2NrmbGoOh5KupAw2w8E547nJjq9AvPwfXvBgRCIBnvzaTeaePom9O6yyFllBKMSDPd8zaZHy5cAhzJvQ7Jq9lklYioZRyAo8CFwCTgKuVUunjnBN6DHEZPDET4TmT+kcCnLGs3VvOa2sPcMjGR23FakmEdHKNAWN57tNdHCiv4/63tvDMxzsTnmcNkJdURbtd6mysmUAoRGMgxJaDVRw3JA+tDSvNmrIba0nEil0opNlfXsfu0tqmPbHj3E0Kl1PhjwTKQ1G7tLXI/lXwSCF8+gic+A349jKYdGlcUGBc/xx+cuHEyP1kYwaJOFZtMu6/YipP3XDiMXktk3RzN80EirTWOwCUUguAy4BNnToqoUvRESvy2EnMH3M/x+eONPpzqKaUTmsAs94f5L/rD/K3ZbspHF7At8+K3qHsmY93UlkfiNw/WtPAwLyMpMd6uLKen/17Iy+u3Mf6/RUA3HDKSNtzGyyTe0l1yyIRDGl2Ha2hMRhi0sDcuMfBLrup6fbqPWV84bFPIveLjlQzZXBevCXhMHaBM2Mf/mCoxRoJY4ABWP2MYUFk9YOvvgqjz2r5eWE6av3fmiB3VyWtLAlgMGDdwWNf+JjQg3l3y2GO+7+F1DQEWj4ZoxK6vcROYrUxr53tdZEVFok+2U2ui399+xQeClfjXvrIx9z0/GpW7irj4Xe28diS7VHXMAWid5axJ8Huo7VtGmtVvfF+W/O+zQpvgOJYSyK8X7V1vnt0SRGlYbdU3xwv8786I+6acdlNls/u3xYXE8Dnhw2XVjAYn93UK9PD0bBwtRyT0JznWAGPzYb//ACGzoR5S1ovEB3s2otNhTUXEOmUytpW0k0kWoVSap5SaqVSamVxcXFnD0dIMfe/tZWq+gA7S2padf6hyubdPK0hNiZR2xi9ys7PbLIkrP7tAXk+eodFo84fpE+2l1U/O4dhBZnsPmo//rJaYxJO9HhLlNUa4tBSNtCB8jr+3wtro+5bqWsMxV3nxZX7WLGzFDCKu0bYTHojemdG3f/ugs8ixYc7Y4Tv88PVQHzcItPjZGCej8OVhkgkjElUHYbXb2Wr9wae8PzO8Bdd+Txc9xpk9Un85mMwX72j3E2xAfQbTxnB8jvnMD6BS7IrkW4isR8Yark/JHwsCq31fK11oda6sG/fvsdscB3NXz7e2eqJrydjBgXtGs9Z2Vtay/bi6qhYQKzrSWvNw4u3sTW8L0AiYmMSNeFV9vDemTzwpamM6pOFN9xvyGpJQLT75cuFQ8j2usjLcHOkyj718qzx/XA5lK0lobVm7J1v8pcEcYZnP9kVqZ62c7Ot2l3GiNv/w6GKem7751qW7yqNPLYvJoPLdDfFNpYzz8vLcBPrVfnD1dPi3FBawzf/uhKtNat3l0WOD87PYJtpSVhE2KHg8hMG0z/Px+HK+nDjREtMQmsoegdemQd/mAafPUf1uMtYNf3XcNOnMPHijpvtOwiXw0G/HF9nD6NDSDeRWAGMVUqNVEp5gKuA1zp5TClhV0kNd7++ie+/sKbN13hg4VYK713cIT74oiNVVNbHuyu01izadLhV/XhShdmfp7o+2uWjtabOssI/7f4lzHnwfQ5bLInYSubqhgC/W/w51z65tNnXjI1JmK6uG04ewRUzhqCUipwTmyljTSs1s6LyM91xK3eA0X2zePjqaQzplcHu0lq01tz9+kZ++up6QiFNea0ff1Dzizfsw3LLdh61fa/md+KFFXsAWLL1SNzrx4pEojTcHSXG6j83w42yTMbXzhrGpccPpPfRFcx2bMJcn3tpxEGIhkCI6oYA50zqz7fOGM304b3YGhYJU/B/d+VUttxzAb2yPAzI9REIaY7WNBoxCacDKg/CXy6Av30BihbDhIvg5uX0vvYpZlx6EzjbFlY1fzKqg6ISsVc5lpsCpZq0ClxrrQNKqVuAhYATeFprvbGTh9XhHKmq50cvrQOSK9KKxczL3l5cw5h+2e0a09yHPmDCgBzeuvX0yLGKOj9/W7qb3y7cygNfmsoVM4a06zXaipnbHitiv1+8jYff2cbGu8+LsjKKjlRHbtc1RtcClNUY1zADuFprVuwqY+bI6GZ3ZsvqlT+dy3f+/lnEFWRt2Gama/bONmIKppsmSiTC9RW5GW7224jE2H45ZHtdDO+dxe6jNTzw9lb+8vEuAEb3zebUMYYLxZlgpbyzpMn6sGYcNQRC+NzOiJVTXNUQFYMYmOeLG48puI0xqbvbjlSjFOR4XZH4BIBbN8DL32DmhpdY4IFa7cVFACchavDhmj+K+1z9mcQYTsjsx0dHqlhbPoI7Xl7LtbNHAJDpcRkWw84POWn/Ir7hLKHx0+2cXXGQyf718PByUA64+PdwwjXg6pjUVZNUGSDJNvZLZ9JKJAC01m8Cb3b2OFLJUx/tjJj9Ob62/wm8LgcNgRDvf16cUCRqGwM0+EPkZ7qp94dsi8LMyWFLjAtmzoPvURJuO5DMHgmtZffRGrK8rjh3TSzmqqwyJjD7/LLdgLERkNk+A+DDbSWR29PuWcSrN5/CCUPzKa9tZPHmw1HXfG7pbn7+7408eV0hcyf1jzzPDKzmZ7jJz3Sz6aDxGTktqY7mZ5LlMf6GueG/pbl7GTSJRF6G2zbF1RSW4b0zef/zYjbsr+TKwqFsPlTJvz7bH/Fp2xWNaa2j4hjltU0TeGWdH5/bGYmPlFQ3RHVqHdU3i0+3H6XoSDV9sj3kZ3qoDwe1Y//W5bV+w9UU8pO59z3+6P4DmTQwa9NOCJSzc/LyFnf6AAAgAElEQVQtPLrGzxmOtWgUDdpNHV5OaaziIudS8na8CzvgVOADL1Suy6Sx9gJucGZx3MpX4JNdsH8l44CfuoFP4OtAuaMXFN4As/4XekdnhrWXgnCyQP/cjnEJxV4n2c2G0pm0E4megDVLpzSmoAmMFfNraw5w1YlDE37ZQiEdCb69se4AXz/VPuXx6vlLWbuvgrsumcTdr29ixZ1zI+6RoiPV9M3xRo3nP+sOsmJXKV+ZPTwiEKnijN++R4bbyeZ7zm/2PNPdFJu9Y05m5bV+Xlm9D5/bQb0/FCd2j7xbxJPXF/K1Z1awOlwoZgZFzXMPVESvqk3LxOlQeF2OyGdk/XM0hid9UxTMQLa5exlY3E0JOnqaIjGsoCn4e8vZY3hj3UF+89YW1u0zUlqdSlFR50cpyPUZ1zpS1RAVULdaU5V1DfTzBXFgfEb7S5ssjkzqmZ5TwV4Oc8PvXqJKZ3Df1afx7zUHyKSeWu0jnyr6qXLcBOinyjnduQce+QH9y/dwSfjtbcuczdjL72BPYDIvrVrOS8Ezot9cuHHuw1dO5bLjB6CrDrDgyQfwVOzgnKL/8n/uWvz786D3CLjgforHXMFZv30XDwEy3DBpxEj+fOFs28+tvVx8/EA0cOGUAR1yvSyvi12/vojj7lpIVUOgwxv/dSYiEp1AVX2AYQWZHDc4j82H4ruInvvQBxyqrGfyoFymDetle43i6gYaAyFG9c3isz3lkYpYgLc2HGJnSQ3nTxnA2vAkc/frhk973b5y5kzsTyikufKJTzlvygCundVUkXrz31cD8NmesqjXs4tXtAfTZ17nD1JR66ey3s/Qgkzbc82Gc7FjMCfFzQcrWbz5MDedOZq/L9sTyfYx2XzQ+IxXWyqJTZEw20DE5rkHQkZ/IqUUXpfTIhpNKvHN00bywefFjA23/si2syTcTZaEHR6XA4J+Jrv2c6bjMwapUoasXs2XDu5ggns77uX9+LP7KE6Hg49/9Ts8+DllRDYZLoWvtpHn3RUoNP1VGTmqDg9+vPjJeNwQ+Gtc2VzhDaJ3Oyn3ZNBHVZKpGmAz/MBiwIVeUXzudeBRQT4PDWa4OoJXNX2OwYADck7k6El3ctmr9YxTexk2+n/4v1FTcFosNzsyPC5wulD5w6id/X3ueGMTHvyMUIf4+TVf5NRxRvJJX+Bb557AA29/Dn6Y4e1Y15IVpRSXTk1cNd9WzO9qd6qbEJFIglW7S5k6JL/dpmRlnZ/cDBcFWZ6IJWG4Dmrpl+uNpHBW1QfYUVzNdxd8xnNfm0WvLA9r9pbzxtoDLA0HLL83ZyzfW7CGJz/cwSVTB3H6uL5862+rAKhuiJ/Y1+6r4L2txby4ci8NgRBLtx/louMGRh7PcDsZ1TcrIi7WMXck1iKy835viOKqn87lX5/t54aTR0R9xuZqOdaSMGMCn+0tI6ThlNF9eG3tgTiR2F9eR1mMxeaOyZiK3fTH7HQKRLKYINrXfNrYvuz69UXsDa/SL5hifI7WmIS5LWd+piES2dQyVBUzXB1Gobn0wKvw24WcVF/BSZ7wkz52kpfVn77KQ5/aA5QrH6Bw46cBD1t2VTCmfy6NgRAe5Uej2KaHUBbKphEXdXipx0OD9nB+v3pWHagngwZ8qpGSUB4lOo8zp03gn6sP4lQhcqgjX1XhIUgIxVS1nXV6NO8GT8CPixKdR5+Rx/Pnr59NQ3kd+/S77NN9+VpYMFsqNra6OE3RbMTN53oozpjf0i1njzVEAji/g1b5x5LIhkXHqAL7WCAi0UrW7i3ni49/yi1njeG288a3+TpPfbSTVbvLmDI4j15ZHspr/QSCIX63+HMeXbKd/z19VOTcX7yxKRKEXbT5MBcdN5D5H2znzfWHAGPCmjWyN0N6ZfDqmgO8uuYA914+Jeq1Zo0sYFz/HJ5bavjv//DOtqjx7CipiaQlAozul8XEAblsPFDJ4PwMPr79bKbfs4iq8KTeGGhdd86WKK5qykAyRfGhRZ/z/LI9+IOadzYfxqEUz3ztRGrD6aeVddHZTaaP30wdzc1wk+11A/EB4hdW7o26b/6YzYygitpoEQkGm0TCOunbrRCHFmSy7Cdz6Bd243ldDjz4yaaOQYeWwJ5NnLP1I0737qAfZTiVZb+GEjdMvIjGsRdy1YsHqPD0552fX8mh8gYuvn8JStnXfd0zYwr7Smv5y5Fd/GPeLJ5fvC0qFmPySSiPtYGKuOPTJszglZWr4i+cgIl1hsg5LJFe86ZdUP30cX354HOjhsmanGG1ssD+87x21jCeX7aHuRP7xz2W7pgZb5Ld1AMxJ7Kth5vPr7dDa83izUc4eXRv7gmnMub63JFK2yNVDTz2nlGN+8QHOyLPs2bp7C2tZfJdCwEj6PjP/z0Jn9tJltfFuP45kXTGn766IfKcen+I8yYPYObIAv62bDdj+mazzXJNk7ctu67NmdA/svIrHGG4unJ8Lirr/SzfWcqXn/iUl286iRnD47e+3FlSg9upGNLL3m1kfhZ3v76JZz7ZFffY++FJ5TdvbYkc+3BbCTUNhiVhdTdZg6umSOT5nIxwlVJKJQMyNcG6CnJUHVP6Z7Bu6T7GKY0fF36cZNX5oLaUsuoGHIQor4t3ZZkTmLVvk60bIRigv6cBij6ETa+idn7I5z5DlFkCONy4CybzTnAK+3Rfhk+cwbMbDcE764yzuPX84/EAVwf3Gu5Fh5PM8N8gUXbz4Yp6dpbUMKx3JjOGFzBjeC9bkYi1CE36ZHtsj8diWruN4aC29e2bt+0+k1xLQoZdSrBJWW183OsXl03hpxdNSps9npMhsqudBK57HmadgEMZ+eQuh2r1F+H1dQf57j8+48ZTRkSOme4mMCpftTaKmGLTD03mW8Rj9qjekawViC+i+uvXZnLd08sBuHzaYAqyPLx/21kM6ZXBK5/tZ2ivDK6c31Qn8Ml2w3X19A2FnDGuH+9uMbbkLBxuiESuz01lnT+SGbR0R2mcSIRCmrMeeA+vy8HWey8wrltUwjVPLmPJbWdSUefnlr+v5r7/Oc5WICA+bx/g3c1HIoVsVndT6eE9vOi5Gx+N6FpFhqeBIY8f5fFAPfiAEGB+RGZ4xeriDgH3w3M4cPpClK7vDznXw6gzYUghgVBTxW+cSNSVwZEtcHQb1JTA8vlQddA4wZcHI07j9yUzqNE+rvvSlxg6+WQOlwX44UPvc/kJg/j+VdOY//sP2HKoijnupl5NXypsqiM1W34kYufRGnYdrWFEb6MC2gyaF2R5GFqQydq95c09PfLds/KNU0fy8up9Ue664b0z+eXlUyJxFxVlSRi37TKvrDGYTE+8+81kdN/4rDynQyXVmj2dMGMSkgLbAzETRxxKMeFnbzF3Yj+evD6+G6OdO8bstvlxUdNKL9fnjvipzVYEPndikbAWSg2PCfB+8/RRfLL9KA9+eSqj+2YzcWAuQ3plMGVQXmQyGBZunXDFjCFRojJ1aD5r95aT43Nx9gTDvD9pdG+uLBzKBeFYRW6Gi6r6QCRVNrYiF+DTHUcj4/QHQzzx/vaIb/mNtQfYcriKfWV13PbPtVHPG9Unix2WqvOLjx/IG+uMCffSqYP47/r9VDYEAYVv/1I+nv8iM0f1xbN9GdNUER+GjsOB5iC9GVv4PyzY5WXd3nKG9Cvgs8NBqsjkvi9O456Xl5JBA24CuAjhUkHuvXAEzy1eSaVfca53BwUf/BY+uB98+czJvYThugH+8Re+umcdp3o8lOtsTnr9INTH7Gs94DiY+U3oPQbGXQAuD7+//T8AXDdsNrh9jOkH//r2yRw/xNirwZxMEq2WvS4HTodRsDdpYC6bwsH3F+bN5o/vFrGzuIbdR2s5c7zRNjoznIY7YUAOz319FvvKavnegjWsSSAWvbPig8L5me6owDwY6b0XWGJW1rmvOXdTrkUkrBO+z5L59fJNJ7W7vifdEJHowZh/fNPnuHjzkbhz9hyt5fTfLuHhq07gshOMvoR/X7aHP71vuJJ2WQqfcjPckZWqmV7pczujArqJGJQf3Sn05NF9Iqt3kw9+mLjRmXU1eMbYPqzdWx6JOYCxKv3NFcdH7ud43RRXVUfcPeV10S6CUEjz+8WfR+5vPlgZEQiABxcZtzOoJ7d6H5MdxYxR+8lX1Xytv58j6hBlFRX48DPmsIOfeyvw0UjONj9K+anxenG6XPiCNQT2O3AdCFEAPB26kBd738SWQ1XkZbg5//xzWf+v9fx91x4u6TeIRQcPADBi2hwytxYwbWg+18waxt+W7ua+N7fw/SlzuOeNkWgNS3Ly+Petk2HXx7Dm75y19TnOQMHRsZTnTkDX7GSMYz+lA05j4JgToN9E6DMOMnpBRj6JsE6QdplqdoJr/o0yPU6q6gOcMCw/IhJTh+Yzpl92xBozLYnSGmOhMX5ADk6HYnjvrDj/vxW7+py8TA+xw5k+LPq9RcUkwnXGdu4mqyWRYVNcaBzvftOPuZhMei+MNKb7/ZXaibnKVjGrI1MkrO2VtdZR5+0uNVbEd7yynvMmD+CNdQf5yb/WRx63Wgk5Plckw6baIhLNMTg/g/3ldbYmeiwt7Q3cJ9tLSXUDN5wyktV7yhneO3EcITfDRWVdIOIOiu0e+mFRCSt2lfFgYSXbPnuP0hcX8LC7hEGqBC9+fDTSR1VQoKLjISEcOMpHMbxgAB5fFp6MLFz5+Sw+eoR+Bb2Ye/wIXlxbTHVZMbOG9WLy8YU8dGgGf/p4H5nUM6BvX4blZ0REAprSUK0+d4dD8eg10yP3zTqRDfsrIj7kslq/MeFPvBgmXszPn3+HpfsaefuWC1i2Yi8/etmokH/ulJkMHNv6fmGJKurN123O726KRK7PzYc/OovVe8rwuZ2M7tvUZG9EH+PvVjjCcP99cXpTVby1XiMW6/fDDI7nZ7gjNSmD8zP44zXTmDoksUiYl3DYWBJWkUgUk8jsoi6l1iCB627MmDv/y0mjevO3b0TvM2taENYmbYcq66P6/5upmrWNQX7yr/W8sjquN2GEkG4qEjMtiZZadHzn7DGcNq4vg/OT33MgliW3nYE/qCnI8sS911hyfW6q6v3sKTU+g9LKGso/for1mzZx4rBcBhVt4CPvaoZsKAE3VFf48DtcbAoNp5IsGnBzMG8aw0dN4KEVdezXfXj55zfi8OVHfBYDLa/3hUuMmA9OB9sbN/PEBzu4ZcgYJp84nttCmutOn8iizYeZNjSfl1fvA5pWxmb1s8/tpFemm5vPGhP3fnK8xgS2Ilz1Xji8V6SFtUmZo4CA0wj6WlNgk81/TyT8psOvOZEwvx+5GS6GFmRG6kisW7Karahnj+rNjvsujJr8m7MkrOR4XVTWB8LuJjPOANNtLB9luaQpDvaB6yaRsMZ07KrRuyNSJ9GNCYY0HxXFZ4mYefRWkdhyqCpKJKwum+YEAoze/eYXyWyXEBvUi6VPtrdDBAKMTXNaS36mm/H+zVwYWsbF3qUU7K7EsyfIaUBov4M+rj6sco5jyIU/pajfXOY+ti7uGsvnzaEg08Nry/5rHMiwLxKE6In14uMH8cQHOxhnaU8xIM/HV2cPB4j43E2BNj/TkNZ89vNzba8/YWAOShmZZA4VjsvsM65ztLqB/eV1BIIh++ymJJv9JJosTMs0dsc7K2a78tyYv5VVJPpbOo3GWo92lsStc8dGUlNNcjPcVNYHyHA7I770RO/TNgXW5i1Y3VlWazuR66m7Ie6mHsZne8p4dY0x6VtTL3cW1/D2xvWcNrYPFx43MLL5i8lVJw4l0+PiaZs2z3Mn9Y/4dJssiea/WH06aF/eZqkrh7KdULoTjm6H4s1cv2cdt3i3EdSKd0PT2KaHsCI0nvdCUxmUn4XToRjVN4u5hTPpV+8HokXi+CF5kbbJXztlJKP6tn4jluOG5PHZz86JBPljMQOfpgssMh8107R2SK9MzhjXl/e2FjMg10eOz4U/qAmFNJf88SMOVNRzzqT+kew162TbYW4E093UzGRiWq+5MdXavbKiXWmJMFft1p3zbp07jlvnjos6b/qwXuwrqyPH12RJJMrcc9plN9kISqJJ0mdTWNcdkcB1N+RodQPPfro7cv+eNzbxkwsn4nQo/sey/SIYPz6Xw8Gmg5W8tGof/1i+h12/vijKkgCYPDiPa2cO4+mPd/LN00Yy7/TRaHRkwtxebPjoa1oZk2htbnuzhEJGumbpdkMESndA+R7jX9lOI73TSv4wnL3Gcf/RaTwbPI9JIwaxYlcZS++Yw86SGq59cikhDReEq2NzYlI3B+dn8Notp0bu//yS5Lcs72WTrmlixmfMuI45YcW2+o7lZxdPAjYxc2RBJD7gD4U4EN6LIhAM2VZcx2b/tJXWuJvMavBcmyDzzWeNDhcOJsYUt16ZHo7a9Agz+c0Xj+d/pg2OBL2huYC65Xb4fztrKZGYWrObutNqOxaXVFx3D5btOMpxQ/LI9Li49YU1UcVIT320ky8XDrXdWap/ePX5zuamIrS3Nhziz5ZaBjB+3A6HYtsvL8AV7gNkxRVxN4VFoplAI8RvbtMi/jo4vBEOroGDa+HwJjiyGfyWjY6cXsgfCnlDYdLlUDASeo1s+t+bjS+keewnRmPeJ68/kQPldQzI8zEgz8f8rxby6Y6jXDvLcP9Y32Omx8mvv3hccmNOElM4bzrT6BJqzlctbX8xum82z9w4E4AnPzT+blYrMbqYzmJJtHKFOCjPFxEcO1pKgYXElgTAD8+b0OIYTEsiL9PdrEhkeJycNcFIpTXfc6L9paMD14ktiURutu7U+K45JHDdDdhZUsOV85dyZeFQfnPF8XGBS4jfwtLE7P+/YX9Tcz6zX5IV0y+baMXkiqTAGv70lny0zVoajbWGEET+rYHiraDDXUIzCqD/ZJj+Vegz1sjpLxgNuYNbbL5jdWnkZbijMlfmTuof1WLbykc/Ptu2aKsjUUqx69cXRe5/cfoQFm48zNdPs++Ka4c5UVtFot4fjExo1riB3YRox6L/d0bchkdWdCvcTebueLExidZitSQgfgfE/rneSI2OScR6SjAu2zoJO0sigUjELpS6K91JDHusSOwrM2oW9ob/j/2xAFG7nlnJcDvj9vU18bgckcmmpeCw2xGbAtv0w7zzwonMGlXApY98zC1njeEr4UBtBK3h0HrY+ArseB8OrYNQ2N2V1Q8GnWDs4jXwBBg4FfKGtGuHlfduOzOy+m0tiTqfppLe2V5evunkpJ5jirg1Rbm2MRgReevfpbUrxCyvC5t6tQhmo/fWWRJt+5ma407k+3/71jOoimkC2WRJtBy4bq4tR3dyt7SF7vT+e6xImN1Xm/N31yXYzjHD4+ScSQN4dMl2MtzOqPNcDoVp2De7oVBtKTkfPsjf3Et4tfYatApy+cFXmOIq487A18jLcHP8kPxoV1VNCex4D7Yvge3vQtUBUE4YOgtO/i4MnWmIQu7AxK/bRkb0aX2w+fFrp7No8+EukwZorub9gSYRrPMHwyvwaHdTR70n00htTiT65njZX17XbktCJ4ji52W6yYtJCDAnt1bFJJpxN7mcig9/lLigs7vTVb77raHHikRZTSMnOzYwwNs3bv/ms8b3ZcnW4qgNXaz43E5OGJrP5l+cT0l1A6fdvyTymDVgamtJNNbAmr/Dkl/iq69glkNxas2dRl+hYpjlgqeD5wPHw6H1uA+ug92fwOH1hhsJwJdv9BgafTZMuBiyerfvw+hgLjhuYFQrh3Qn4m4KNv296xqDEashKnDdwe6S5txNC+bN5tMdR1tMaEiEaUm0FMS3ErEkmqkEb7od/ZzY6yTaH6Qn0J2C8j1WJNSRzfzdcx9rj2yhpOb4qMeyw5P7b97aYtuF0zTfMzzOuBYZVj90xJIINMDGV2Hza8Zm7oF6GHEa9XPvY+YjW7nSt5QKv4PjZ87hq59dyR/cjzD+rZ9DMNzwLrM39J0IZ/3UEIZBJ4Cj+6YPHmvMH7T1b1fbGGzyz6fAktCtCFxbC+jagjnukDZcQ7NGtryYMIWxNR1YI2057CyJbrSSbgsSuO4GZJZtBiArUBYXezA7au4orrENSGfE7C9gzUOPuk7VLnh3viEQNUcgZxBMvw4mXgojTsUZ1FSxl2ca5xAIaUbnj2VVaCyT1G72DrqYkdPOMlxJvceIKKQQMzBt3YO6rjFoW0zXUT9+8/uSypbSpgUUCml2/OqiFs42MF1HrRGJSFsOm1Obe18v/u9JXbINeDLIpkPdgKwqI+2xxNmPupLozI/YWEKsCMRmIbmdjsgq9K5ZDo6sfJWZjs04Hl0LTg+MvwCmXWdYAZYvj/k7CYQ0HpcDp1Jc3/hjFHD3tJMYaenDI6QOc8K6/NGPI8cagyFLMV3q3E2pXG82WRKtdzeZFkBzbjCT5tpyNGdJzBwZvxdJd8MplkTXp1+tsUPb5gPl3P2XFVGPxRaE+dzOqPhElI840Mhpzg300Yf4/pgj9F//JrgDFOs8OPMnUHgjZPezHYPDYoV4XQ6UgmoM90IPyRRMCxKmKIcnOmOfa2Mh0FHupvOnDOCZT3Y1n9zQTsyYRBIhiaZiuta4m5pp8NedArdtoTu523qsSAxs3AVAts1Wl7GWQkaMSGS4nUam0cq/wIo/86Q6DG4IHMqHaV9h9sczOEIvdpx5SYvjcDmNlNnYPjutzccX2k+iCdE60Zki0VGpjT+9aCK3nD0mqR5ayRLJbkrGknA2H7i2opqxJLqTu6UtSOC6FSilfgtcAjQC24Ebtdbl4cfuAL4OBIHvaq0Xho+fDzwMOIEntda/TtX4ev9oDSW/Hk+Wiq+KbWkVMLlsMfz+F+CvhdFz+GHd9Wyszeexm69nRN9s7h19OFJF3RJmyqzX5RBh6CQSuVasE53X7YT6QEt1h63G5XQkX0GfJJHspiREwmw70rrAdfg5dpZEN3K3tIXuZEml0pJYBNyhtQ4opX4D3AH8WCk1CbgKmAwMAhYrpcyOY48C5wD7gBVKqde01ptSMThfRibbVD+yiBcJZ8ykYbY0GKf2Mkod5OyNf4Bhs+Di30O/CfR5awub3ttOr/CPPlEFsh3W/kB2OehC6vEkKByzTnRmXKIrFUmZq9kEjQNsMd9ya1bCTYFryW7qzqRMJLTWb1vuLgWuCN++DFigtW4AdiqlioCZ4ceKtNY7AJRSC8LnpkQkAOrwJbQkFCFGqYP0ooqVejxnONbxrOc3AJTlTqTXtS+B12gu98Nzx/O/p49qU4Wx29Jp1Pqzkt/YscPjtM8cs050ZhyqK60QzXVGMoFrM37RXAvzpusn/iy60uckNM+xikl8DXghfHswhmiY7AsfA9gbc7z53XDaSa3KoD8VccdHHniTnb4fR+5X6gxylRG7+FPgEkac8XPO9zbtDudwKPIz29ajyGXpDxS1W1hK814EK4lcK1arocmS6Dp/F3OiTqabiumaak3vIetH8Y1TR3L2xH5c8+dlQNf6nITmaZftrJRarJTaYPPvMss5dwIB4Pn2DtZyzXlKqZVKqZXFxcUtPyEBdSoz4m4aq/bxZ/cD9KaCqVseBODV4MkUhQaRq+rYnTudH/Z7gl8Hrsadmdch7wOaJiKvyxElC+JtOnYkFAkbd1NL28KmE2ZbkRkjEm/wFEukyK81gVfLl/SnF0/i5NF9LA91nc+pIzlzfOu3tu0qtMuS0FrPbe5xpdQNwMXAHN2UYrEfGGo5bUj4GM0cj33d+cB8gMLCwuS6zlmoV03upmuc73COczW56mEy6o/wxYa7WKXHM4Cj/I/zY3yTbmH37mqgtEM3S2lq/eCM+tF1obmoy5No1Ryd3eTscqvj/rk+Ft56emQf7NZgtvBI1CrcShf7OI4JT15XGNkHpLuQsihcOFPpR8ClWutay0OvAVcppbxKqZHAWGA5sAIYq5QaqZTyYAS3X0vV+ADqHRn0VlWMUAc5x2lUVs9ybOHQwDms0uMBOERvHg9eisPti/R4as0PqLWYE0+m2xnzo5Nf4LEikSXhjopJOLqkn338gBzbbUwTYc5vyRTTCU24nI4299pKV1IZk3gEo23dorDpuVRr/S2t9Ual1IsYAekAcLPWxqYHSqlbgIUYKbBPa603pnB81CljhfWe9wdRxw8OvwRidhx1OR2RFUJHThamuynD44yKQ8jv79iRaEJ0RsUknF1SJJLFXAglkwIrdG9Smd00ppnHfgn80ub4m8CbqRpTLI2ODNvjZQNP58fnw4pdpby75QhguCTMLJGObM1gupt8MZaErNKOHYn851ExiS5qSSRLKImYhHxHewZdJ+k7BeRTGXX/4oZ7+W7jLShvDjedOZqnbzgx8pjb6WBArrE3dZa3I2MSTRvD2O0fLHQerqiYhKPLxSTagtnjsFUVw93/4xDowW05AHa7RgHwW/+XeS80lY16JBv0KL5o12bA6eC3V0zlos+PMKZf/L7XbcWceDI8DnE3pQHfOmM0f3p/OxDtVpwxvFdkB8HuTDJxN7EkegY9WiQ+9Z3O1JLRVJAdddy6YjQb8LmcirxMN5edMDj2Mu3CTPqKtSTkB9g53H7BhIhIWGtfrjxxGFeeOKyzhnXMMLObWmM1yTe0Z9Cj3U1OpyNOICB6BdnSdo7txQyGZ3hc0b5x+QV2Ov1yUttbKR0xYxLNLVIi3V979OzRc+jRf+ZEgUjrKso8J1U7TZkb3WS4o9tyiEZ0Pv1ye7JItHyudAXoGfRod1Mikzrakmh96+S24A8a3dcyPNHXF3dT59Mvx9fZQzjmBFuR5q0AjcTNego9WiQSWxLx21W2ppdNWwiYIuF2Uu9vatcpP8BjS+8sD/1zo0Uhvw0NG7s6ZnZTc+1HlFKgdY9tvdHT6NEi0RpLwiyoSp0lYfwqfW5nZAtUEFP+WLPqZ+fEHetKfZo6CjO7qTW1QD3w4+mR9PCYRMs7kpkWRKr2EZUeONoAABCrSURBVGi0WBJKejcJncxXZhsZXMMKEvd7Mr+a4hLtGYglYYPTJnCdaGOa9hJxN3miA9diSHQuvbPa1vq9q5NMqq/dV/TfN5/S7Rrc9XR6tEhYdx77ztlj+OO7RUC0eJhuplRZEoGgfZ2EuJs6jw13n9ehrVe6G+ZHYxeTmDo0/xiPRkg1PdvdFP6S33LWGH5w7vim48cyBTbUZEk4xN2UFmR7XWR4ulcnz1QgOtoz6NEiYU7EsQJgvW9aFala2Sesk5BfoJCmmL8FiUn0DHq0SJjExibsLIlAMrvJJ0EwUnEd426S35+Q5shXtGcgIkFTJ9bIfUv8YVC+0U481W2ifS7JbhK6CNKWo0fRowPXZg5GnCVhmawfuGIq/91wkMmDOm5faysvfeskFm48hMMR69ASlRDSG0mu6Bn0aJEwiRMJS0wiL9PNVTNT1/2zcEQBhSMKgGgfr7ibhHTF/GrKd7Rn0KMNxnAvM5xx7qbO+fZLE1ihKyGB655BjxYJE3czgetjiewnIXQl5CvaM+jRImHGJGJFobMKqZS4m4QuQGQ/CfmS9gh6tEiYvfNjm/d1VmO36P0k5AcopDfyDe0Z9GiRMEmXbp9iSQhdAXMBIwWfPYOeLRJhf1O6fNWtWiW/PyHdke9ozyDlIqGU+oFSSiul+oTvK6XUH5RSRUqpdUqp6ZZzr1dKbQv/uz7VY0s3rC4mcTcJ6YrEJHoWKa2TUEoNBc4F9lgOXwCMDf+bBTwOzFJKFQB3AYUYa/xVSqnXtNZlqRqfDpsS6fJdj8pu6tk2ntAFSJffjZBaUj0V/Q74EU2JRACXAX/VBkuBfKXUQOA8YJHWujQsDIuA81M8PiB9Vu3SKlzoCjRtOtSpwxCOESkTCaXUZcB+rfXamIcGA3st9/eFjyU6njJ0mu2NEuVukh+gkPbIl7Qn0C53k1JqMTDA5qE7gZ9guJo6HKXUPGAewLBhbW+ZYYqEOSH/+PwJfFRU3N7htZnoYrpOG4YgtAr5jvYM2iUSWuu5dseVUscBI4G14TS5IcBqpdRMYD8w1HL6kPCx/cCZMcffS/C684H5AIWFhW22ByIxifD9m84czU1njm7r5dpNdCBQfoFCemKmvkrgumeQEneT1nq91rqf1nqE1noEhutoutb6EPAacF04y2k2UKG1PggsBM5VSvVSSvXCsEIWpmJ8saTLd132kxC6EvId7Rl0RhfYN4ELgSKgFrgRQGtdqpS6B1gRPu8XWuvSVA4k/WISTcgqTUhXmgLX8h3tCRwTkQhbE+ZtDdyc4LyngaePxZgAJg7M5e1Nh+mf6ztWL9ksURXXnTgOQRAEkx69n8R354zljPF9mTasV2cPBRB3k9BFkGK6HkWPLtlyOhTT00QgQNxNQtdANh3qWfRokUg3RBiEroBkN/UsRCTSCHE3CV0J+Y72DEQk0gjrykxWaUK60tTgr3PHIRwbRCTSFNEIId2R/SR6BiISaYQ0+BO6EvIN7RmISKQR0e6mThyIIDSDFNP1LEQk0ghp3SR0JUQjegYiEmmE7EwndAUkBbZnISKRRlhdTOJuEgQhHRCRSCOi6yREJYT0JBKTkJVMj0BEIq2QBn9C10G+oz0DEYk0ItrdJD9BIT1pKqaT72hPQEQijVDRhRKCkNaIRvQMRCTSiKgMWPkBCmmOfEV7BiISaYT0bhK6BirqP6F7IyKRRoi3SehSpNn2v0JqEJFIU8SQENIV+W72LEQk0ghr3rm4m4R0RwyJnoGIRBohsiB0BeR72rMQkUgjrNaDGBJCuqPFlOgRiEikEbKfhNAV+P1VJzB7VAF9sj2dPRThGJBSkVBKfUcptUUptVEpdb/l+B1KqSKl1Fal1HmW4+eHjxUppW5P5djSEassSFscIV05eXQfFsw7CZdT1pg9AVeqLqyUOgu4DJiqtW5QSvULH58EXAVMBgYBi5VS48JPexQ4B9gHrFBKvaa13pSqMaYbKsrdJCohCELnkzKRAG4Cfq21bgDQWh8JH78MWBA+vlMpVQTMDD9WpLXeAaCUWhA+tweJhOV25w1DEAQhQirtxXHAaUqpZUqp95VSJ4aPDwb2Ws7bFz6W6HgcSql5SqmVSqmVxcXFKRh65yBtOQRBSDfaZUkopRYDA2weujN87QJgNnAi8KJSalR7Xs9Eaz0fmA9QWFjYbXIsHOJuEgQhzWiXSGit5yZ6TCl1E/CK1loDy5VSIaAPsB8Yajl1SPgYzRzvEYguCIKQbqTS3fQqcBZAODDtAUqA14CrlFJepdRIYCywHFgBjFVKjVRKeTCC26+lcHxph6S9CoKQbqQycP008LRSagPQCFwftio2KqVexAhIB4CbtdZBAKXULcBCwAk8rbXemMLxpR1KMgoFQUgzUiYSWutG4CsJHvsl8Eub428Cb6ZqTOmO2BGCIKQbsnZNIyRYLQhCuiEikUZIlbUgCOmGiEQaIYFrQRDSDRGJNEK8TYIgpBsiEmmEiIQgCOmGiEQaIe4mQRDSDRGJNEIsCUEQ0g0RiTRC9rUWBCHdEJFII0QiBEFIN0Qk0ggxJARBSDdEJNIIqbgWBCHdEJEQBEEQEiIiIQiCICREREIQBEFIiIiEIAiCkBARCUEQBCEhIhKCIAhCQkQkBEEQhISISAiCIAgJEZEQBEEQEiIiIQiCICREREIQBEFISMpEQil1glJqqVJqjVJqpVJqZvi4Ukr9QSlVpJRap5SabnnO9UqpbeF/16dqbIIgCELrcKXw2vcDd2ut/6uUujB8/0zgAmBs+N8s4HFgllKqALgLKAQ0sEop9ZrWuiyFYxQEQRCaIZXuJg3khm/nAQfCty8D/qoNlgL5SqmBwHnAIq11aVgYFgHnp3B8giAIQguk0pK4FViolHoAQ4xODh8fDOy1nLcvfCzRcUEQBKGTaJdIKKUWAwNsHroTmAN8X2v9slLqy8BTwNz2vJ7ldecB8wCGDRvWEZcUBEEQbGiXSGitE076Sqm/At8L3/0n8GT49n5gqOXUIeFj+zFiFtbj7yV43fnAfIDCwkKd/MgFQRCE1pDKmMQB4Izw7bOBbeHbrwHXhbOcZgMVWuuDwELgXKVUL6VUL+Dc8DFBEAShk0hlTOKbwMNKKRdQT9g9BLwJXAgUAbXAjQBa61Kl1D3AivB5v9Bal6ZwfIIgCEILpEwktNYfATNsjmvg5gTPeRp4OlVjEgRBEJJDKq4FQRCEhIhICIIgCAkRkRAEQRASIiIhCIIgJEREQhAEQUiIiIQgCIKQEBEJQRAEISEiEoIgCEJCRCQEQRCEhIhICIIgCAkRkRAEQRASIiIhCIIgJEREQhAEQUiIiIQgCIKQEBEJQRAEISEiEoIgCEJCRCQEQRCEhIhICIIgCAkRkRAEQRASkrI9roW28di108nwODt7GIIgCICIRNpx4XEDO3sIgiAIEcTdJAiCICREREIQBEFISLtEQin1JaXURqVUSClVGPPYHUqpIqXUVqXUeZbj54ePFSmlbrccH6mUWhY+/oJSytOesQmCIAjtp72WxAbgC8AH1oNKqUnAVcBk4HzgMaWUUynlBB4FLgAmAVeHzwX4DfA7rfUYoAz4ejvHJgiCILSTdomE1nqz1nqrzUOXAQu01g1a651AETAz/K9Ia71Da90ILAAuU0op4GzgpfDznwUub8/YBEEQhPaTqpjEYGCv5f6+8LFEx3sD5VrrQMxxQRAEoRNpMQVWKbUYGGDz0J1a6393/JBaRik1D5gHMGzYsM4YgiAIQo+gRZHQWs9tw3X3A0Mt94eEj5Hg+FEgXynlClsT1vPtxjQfmA9QWFj4/9s7uxAryjCO//6IrZFSiSKGkW4IIRG2mBiJRNGXRSJ4sVQU1FUfUESUIoRd7EVBZkUkfZj0YfQpiChkKXSV5ua6WpZa2YWYi4RWN1H5dPE+6x6OM+tx3fmg8/xgODPvjMxv/+Oed+c97zzHRuAXBEEQtEBRD9NtANZJWglcAswEdgACZkqaQeoEuoG7zMwkbQOWkD6nuA9o6S6lt7f3mKRfRug5CTg2wn9bFHV0gnp6hVNr1NEJ6unVTk6XtXKQzEb+h7ikxcDLwGTgONBnZrf4vuXA/cA/wGNmttnbFwKrgDHAGjPr8fZOUgcxEdgF3GNmf41YrjX/nWY258xHlkcdnaCeXuHUGnV0gnp6hdPpnNOdhJmtB9bn7OsBejLaNwGbMtp/Is1+CoIgCGpCPHEdBEEQ5NLuncRrVQtkUEcnqKdXOLVGHZ2gnl7h1MQ5fSYRBEEQ/L9p9zuJIAiCYBjaspPIKzJYkcshSXsk9Una6W0TJW2RdMBfLy7YYY2kAUl7G9oyHZR4ybPrl9RVotMKSYc9qz6fKTe4L7Og5Cg7XSppm6TvvLDlo95edVZ5XpXlJWmcpB2SdrvTM96eWchTUodvH/T900t0Wivp54acZnt7KdfPzzVG0i5JG327spxOw8zaaiFNvf0R6ATOA3YDsyr0OQRMamp7Dljq60uBZwt2WAB0AXvP5AAsBDaTnnmZB2wv0WkF8ETGsbP8OnYAM/z6jinAaSrQ5esTgP1+7qqzyvOqLC//mcf7+lhgu2fwIdDt7auBB339IWC1r3cDHxSQU57TWmBJxvGlXD8/1+PAOmCjb1eWU/PSjncSmUUGK3ZqZhGpyCGUUOzQzL4EfmvRYRHwtiW+Ij0pP+pfp5fjlEdeQcnRdjpiZt/4+h/APlKNsaqzyvPKo/C8/Gf+0zfH+mLkF/JszPBj4EZJKskpj1Kun6RpwO3AG749XMHTwnNqph07ibwig1VhwGeSepVqUgFMMbMjvv4rMKUCrzyHqvN7xG/91zQMw5Xu5Lf5V5P+Gq1NVk1eUGFePoTSBwwAW0h3LHmFPE85+f4TpMKfhTqZ2WBOPZ7TC5I6mp0yfEeTVcCTwEnfHq7gaSk5NdKOnUTdmG9mXaTv2HhY0oLGnZbuKyudglYHB+dV4HJgNnAEeL4KCUnjgU9IlQR+b9xXZVYZXpXmZWb/mtlsUi22ucAVZZ4/i2YnSVcCy0hu15AqPjxVlo+kO4ABM+st65xnSzt2EsMVHywdMzvsrwOkp9fnAkcHb2v9daACtTyHyvIzs6P+S34SeJ2hIZLSnCSNJb0Rv2dmn3pz5VlledUhL/c4DmwDrsULeWac95ST77+QVPizaKdbfbjOLJUBeotyc7oOuFPSIdLQ9w3Ai9QkJ2jPTuJrvMigzxjoJhUkLB1JF0iaMLgO3Ez6tr8NpCKHcBbFDkeZPIcNwL0+82MecKJhqKVQmsaDF5OyGnTq9pkfMxgqKDna5xfwJrDPzFY27Ko0qzyvKvOSNFnSRb5+PnAT6bOSwUKecHpWgxkuAbb6XVnRTt83dPAijf035lTo9TOzZWY2zcymk96LtprZ3VSYU5Zk2y2kWQv7SWOkyyv06CTNMtkNfDvoQhpj/AI4AHwOTCzY433ScMTfpPHPB/IcSDM9XvHs9gBzSnR6x8/ZT/plmdpw/HJ3+gG4rSCn+aShpH6gz5eFNcgqz6uyvICrSIU6+0lvuk83/J/fQfqw/COgw9vH+fZB399ZotNWz2kv8C5DM6BKuX4NftczNLupspyal3jiOgiCIMilHYebgiAIghaJTiIIgiDIJTqJIAiCIJfoJIIgCIJcopMIgiAIcolOIgiCIMglOokgCIIgl+gkgiAIglz+AxljKxglrJNIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(scores)\n",
    "plt.plot(average_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.save(agent.model.state_dict(), \"lunar_lander_ppo_model.pth\")\n",
    "# torch.save(agent.model_old.state_dict(), \"lunar_lander_ppo_model_old.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path=\"trained_models/ppo/lunar_lander_ppo_model.pth\"\n",
    "model_old_path=\"trained_models/ppo/lunar_lander_ppo_model_old.pth\"\n",
    "\n",
    "agent.model.load_state_dict(torch.load(model_path))\n",
    "agent.model_old.load_state_dict(torch.load(model_old_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "env = gym.wrappers.Monitor(env, \"./vid\", video_callable=lambda episode_id: True,force=True)\n",
    "\n",
    "for episode in range(1model_path=\"trained_models/ppo/Pendulum-v0_ppo_model.pth\"\n",
    "model_old_path=\"trained_models/ppo/Pendulum-v0_ppo_model_old.pth\"\n",
    "\n",
    "agent.model.load_state_dict(torch.load(model_path))\n",
    "agent.model_old.load_state_dict(torch.load(model_old_path))):\n",
    "    state = env.reset()\n",
    "    score = 0\n",
    "    input()\n",
    "\n",
    "    for t in range(700):\n",
    "        action, log_prob = agent.act(torch.FloatTensor(state))\n",
    "        next_state, reward, done, _ = env.step(action.item())\n",
    "        env.render()\n",
    "        \n",
    "        score += reward\n",
    "        \n",
    "        if done:\n",
    "            print(score)\n",
    "            break;\n",
    "        \n",
    "        state = next_state\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
